﻿using System;
using System.Collections.Generic;

using System.Text;
using System.Text.RegularExpressions;
using System.Net;
using System.Threading;
using System.IO;
using System.Web;
using System.Collections;
using System.Net.Sockets;
using MSL.Utility.Text;

namespace MSL.Utility.Html
{
    /// <summary>
    /// HTML相关类（不含正则）<br/>
    /// add by mashanlin 2013-6-25 <br/>
    /// 修改记录：<br/>
    /// 1.2013-06-27 mashanlin 增加获取网页源码<br/>
    /// 2.2013-07-02 mashanlin 增加html清除样式<br/>
    /// 3.2014-03-12 mashanlin 增加POST方式获取源码<br/>
    /// 4.2014-05-01 mashanlin 静态方法修改成为单例<br/>
    /// </summary>
    public class HtmlHelper
    {
        #region 变量

        //图片正则
        private const string REG_IMG = "<img[^>]+src=\\s*(?:'(?<src>[^']+)'|\"(?<src>[^\"]+)\"|(?<src>[^>\\s]+))\\s*[^>]*>";
        //链接
        private const string REG_LINK = @"<a(.*?)href=\\s*(?:'(?<href>[^']+)'|""(?<href>[^""]+)""|(?<href>[^>\s]+))\\s*[^>]*>";
        //HTML标题
        private const string REG_TITLE = "<Title[^>]*>(?<Title>[\\s\\S]{5,})</Title>";
        private static HtmlHelper _instance;
        private static readonly object _lock = new Object();

        #endregion

        #region 单一实例
        /// <summary>
        /// 
        /// </summary>
        private HtmlHelper()
        {

        }

        /// <summary>
        /// 
        /// </summary>
        ~HtmlHelper()
        {
            Dispose();
        }
        /// <summary>
        /// 返回唯一实例
        /// </summary>
        public static HtmlHelper Instance
        {
            get
            {
                return GetInstance();
            }
        }
        /// <summary>
        /// 获得单一实例
        /// </summary>
        /// <returns></returns>
        private static HtmlHelper GetInstance()
        {
            if (_instance == null)
            {
                lock (_lock)
                {
                    if (_instance == null)
                    {
                        _instance = new HtmlHelper();
                    }
                }
            }
            return _instance;
        }

        /// <summary>
        /// 
        /// </summary>
        public void Dispose()
        {
            //Dispose(true);
            GC.SuppressFinalize(this);
        }
        #endregion

        #region HTML相关操作
        /// <summary>
        /// 字符串转换成js
        /// </summary>
        /// <param name="input">源码</param>
        /// <returns></returns>
        public string ConvertToJS(string input)
        {
            StringBuilder sText = new StringBuilder();
            Regex re;
            re = new Regex(@"\r\n");
            string[] arr = re.Split(input);
            foreach (string strLine in arr)
            {
                sText.Append("document.writeln(\"" + strLine.Replace("\"", "\\\"") + "\");\r\n");
            }
            return sText.ToString();
        }
        /// <summary>
        /// 替换掉 nbsp
        /// </summary>
        /// <param name="input"></param>
        /// <returns></returns>
        public string ReplaceNbsp(string input)
        {
            if (input == null)
            {
                return "";
            }
            if (input.Length > 0)
            {
                input = input.Replace(" ", "");
                input = input.Replace("&nbsp;", "");
                input = "&nbsp;&nbsp;&nbsp;&nbsp;" + input;
            }
            return input;
        }
        /// <summary>
        /// 将字符串格式化成HTML代码
        /// </summary>
        /// <param name="input"></param>
        /// <returns></returns>
        public string StringToHtml(string input)
        {
            StringBuilder builder = new StringBuilder(input);
            input.Replace("&", "&amp;");
            input.Replace("<", "&lt;");
            input.Replace(">", "&gt;");
            input.Replace("rn", "<br>");
            input.Replace("n", "<br>");
            input.Replace("t", " ");
            input.Replace(" ", "&nbsp;");
            return builder.ToString();
        }
        #endregion

        #region 字符串编码
        /// <summary>
        /// 返回 HTML 字符串的编码结果
        /// </summary>
        /// <param name="input">字符串</param>
        /// <returns>编码结果</returns>
        public string HtmlEncode(string input)
        {
            return HttpUtility.HtmlEncode(input);
        }

        /// <summary>
        /// 返回 HTML 字符串的解码结果
        /// </summary>
        /// <param name="input">字符串</param>
        /// <returns>解码结果</returns>
        public string HtmlDecode(string input)
        {
            return HttpUtility.HtmlDecode(input);
        }

        #endregion

        #region 判断文件流是否为UTF8字符集

        /// <summary>
        /// 判断文件流是否为UTF8字符集
        /// </summary>
        /// <param name="inputStream">文件流</param>
        /// <returns>判断结果</returns>
        public bool IsUTF8(FileStream inputStream)
        {
            int i;
            byte octets;  // octets to go in this UTF-8 encoded character 
            byte chr;
            bool bAllAscii = true;

            octets = 0;
            for (i = 0; i < inputStream.Length; i++)
            {
                chr = (byte)inputStream.ReadByte();

                if ((chr & 0x80) != 0) bAllAscii = false;
                if (octets == 0)
                {
                    if (chr >= 0x80)
                    {
                        do
                        {
                            chr <<= 1;
                            octets++;
                        }
                        while ((chr & 0x80) != 0);

                        octets--;
                        if (octets == 0) return false;
                    }
                }
                else
                {
                    if ((chr & 0xC0) != 0x80) return false;
                    octets--;
                }
            }
            if (octets > 0 || bAllAscii)
            {
                return false;
            }
            return true;
        }
        #endregion

        #region 将 Stream 转化成 string
        /// <summary>
        /// 将 Stream 转化成 string
        /// </summary>
        /// <param name="s">Stream流</param>
        /// <returns>string</returns>
        public string ConvertStreamToString(Stream s)
        {
            string patternsult = "";
            using (StreamReader sr = new StreamReader(s, Encoding.UTF8))
            {
                Char[] read = new Char[256];
                int count = sr.Read(read, 0, 256);
                while (count > 0)
                {
                    string input = new String(read, 0, count);
                    patternsult += input;
                    count = sr.Read(read, 0, 256);
                }
            }
            return patternsult;
        }
        #endregion

        #region 压缩HTML输出
        /// <summary>
        /// 压缩HTML输出
        /// </summary>
        public string ZipHtml(string html)
        {
            html = Regex.Replace(html, @">\s+?<", "><");//去除HTML中的空白字符
            html = Regex.Replace(html, @"\r\n\s*", "");
            html = Regex.Replace(html, @"<body([\s|\S]*?)>([\s|\S]*?)</body>", @"<body$1>$2</body>", RegexOptions.IgnoreCase);
            return html;
        }
        #endregion

        #region 加载文件块/JS/CSS
        /// <summary>
        /// 加载文件块
        /// </summary>
        public string LoadFile(string path, System.Web.UI.Page p)
        {
            return @p.ResolveUrl(path);
        }
        /// <summary>
        /// 加载CSS样式文件
        /// </summary>
        public string LoadCSS(string cssPath, System.Web.UI.Page p)
        {
            return @"<link href=""" + p.ResolveUrl(cssPath) + @""" rel=""stylesheet"" type=""text/css"" />" + "\r\n";
        }
        /// <summary>
        /// 加载javascript脚本文件
        /// </summary>
        public string LoadJS(string jsPath, System.Web.UI.Page p)
        {
            return @"<script type=""text/javascript"" src=""" + p.ResolveUrl(jsPath) + @"""></script>" + "\r\n";
        }
        #endregion

        #region POST方式提交数据

        /// <summary>
        /// POST方式提交数据
        /// 1：单线程程序中推荐使用 Dictionary, 有泛型优势, 且读取速度较快, 容量利用更充分.
        /// 2：多线程程序中推荐使用 Hashtable, 默认的 Hashtable 允许单线程写入, 多线程读取, 对 Hashtable 进一步调用 Synchronized() 方法可以获得完全线程安全的类型. 而 Dictionary 非线程安全, 必须人为使用 lock 语句进行保护, 效率大减.
        /// 3：Dictionary 有按插入顺序排列数据的特性 (注: 但当调用 Remove() 删除过节点后顺序被打乱), 因此在需要体现顺序的情境中使用 Dictionary 能获得一定方便.
        /// </summary>
        /// <param name="postUrl">要提交的页面</param>
        /// <param name="paramers">参数列表</param>
        public string PostData(string postUrl, Dictionary<string, string> paramers)
        {
            string remoteInfo = String.Empty;
            if ((postUrl + "").Length == 0)
            {
                throw new ArgumentNullException();
            }
            System.Net.WebClient client = new System.Net.WebClient();
            System.Collections.Specialized.NameValueCollection postVars = new System.Collections.Specialized.NameValueCollection();
            foreach (KeyValuePair<string, string> kvp in paramers)
            {
                postVars.Add(kvp.Key, kvp.Value);
            }
            try
            {
                byte[] byRemoteInfo = client.UploadValues(postUrl, postVars);
                //获取返回信息
                remoteInfo = System.Text.Encoding.Default.GetString(byRemoteInfo);
            }
            catch
            {
                throw new Exception();
            }
            return remoteInfo;
        }

        #endregion

        #region 获得链接

        /// <summary>
        /// 多个链接
        /// </summary>
        /// <param name="input">输入内容</param>
        public List<string> GetLinks(string input)
        {
            return CRegex.GetList(input, REG_LINK, "href");
        }
        /// <summary>
        /// 单个链接
        /// </summary>
        /// <param name="input">输入内容</param>
        public string GetLink(string input)
        {
            return CRegex.GetText(input, REG_LINK, "href");
        }
        /// <summary>
        /// 获取链接中包含的内容
        /// </summary>
        /// <param name="linkTag">链接标签 如：<a href="http://job.cnblogs.com/cate-cpp_programmer/" target="_blank">C++招聘</a></param>
        /// <returns>返回 C++招聘</returns>
        public string GetTextByLink(string linkTag)
        {
            return CRegex.GetText(linkTag, @"<a(?:\s+[^>]*)?>((?:[\s\S]*?))</a>", 1);
        }
        /// <summary>
        /// 在现有链接中用关键字过滤
        /// </summary>
        /// <param name="listA"></param>
        /// <param name="listKey"></param>
        /// <returns></returns>
        public Dictionary<string, string> GetLinksByKey(Dictionary<string, string> listA, List<string> listKey)
        {
            if (listKey == null)
            {
                return listA;
            }
            Dictionary<string, string> dic = new Dictionary<string, string>();
            string key = "";
            foreach (string s in listKey)
            {
                key += "([\\s\\S]*" + ReguSpeciChar(s) + "[\\s\\S]*)|";
            }
            key = (key != "") ? key.Substring(0, key.Length - 1) : "[\\s\\S]+";
            foreach (KeyValuePair<string, string> kvp in listA)
            {
                if (Regex.IsMatch(kvp.Value, key))
                {
                    if (!dic.ContainsKey(kvp.Key))
                    {
                        dic.Add(kvp.Key, kvp.Value);
                    }
                }
            }
            return dic;
        }
        /// <summary>
        /// 
        /// </summary>
        /// <param name="input"></param>
        /// <returns></returns>
        private string ReguSpeciChar(string input)
        {
            string[] arr = new string[] { ".", "$", "^", "{", "[", "(", "|", ")", "*", "+", "?", "#" };

            foreach (string s in arr)
            {
                input = input.Replace(s, "\\" + s);
            }
            return input;
        }
        #endregion

        #region 是否存在脚本链接

        /// <summary>
        /// 是否存在脚本链接
        /// </summary>
        /// <param name="htmlCode">源码</param>
        /// <returns></returns>
        public bool IsExistsScript(string htmlCode)
        {
            return Regex.IsMatch(htmlCode, @"<script[^>]+src\s*=\s*(?:'(?<src>[^']+)'|""(?<src>[^""]+)""|(?<src>[^>\s]+))\s*[^>]*>", RegexOptions.Multiline | RegexOptions.IgnoreCase);
        }
        #endregion

        #region 去除标签
        /// <summary>
        /// 过滤Html标记  保留p,img,br标记
        /// </summary>
        /// <param name="htmlCode">HTML字符串</param>
        /// <returns>得到包含p，img,br的HTML</returns>
        public string ClearHtml(string htmlCode)
        {
            return ClearHtml(htmlCode, "p|img|br");
        }
        /// <summary>
        /// 过滤Html标记
        /// </summary>
        /// <param name="htmlCode">源码</param>
        /// <param name="leaveTags">保留标签，如：img|p|br|a</param>
        /// <returns></returns>
        public string ClearHtml(string htmlCode, string leaveTags)
        {
            if (leaveTags != null && leaveTags != "")
            {
                //保留p,img,br标记   <(?!|p|img|br)(.[^>]*)>
                htmlCode = Regex.Replace(htmlCode, @"<(?!" + leaveTags + ")(.[^>]*)>", "", RegexOptions.IgnoreCase);
            }
            else
            {
                //去除所有HTML标签
                htmlCode = Regex.Replace(htmlCode, @"<(.[^>])*>", "");//@"<(\/\s*)?!?((\w+:)?\w+)(\w+(\s*=?\s*(([""'])(\\[""'tbnr]|[^\7])*?\7|\w+)|.{0})|\s)*?(\/\s*)?>",//清除所有html标签  
                htmlCode = htmlCode.Replace("<", "");
                htmlCode = htmlCode.Replace(">", "");
            }

            string[] Regexs ={
                                @"<script[^>]*?>.*?</script>",
                                @"([\r\n])[\s]+",
                                @"&(quot|#34);",
                                @"&(amp|#38);",
                                @"&(lt|#60);",
                                @"&(gt|#62);",
                                @"&(nbsp|#160);",
                                @"&(iexcl|#161);",
                                @"&(cent|#162);",
                                @"&(pound|#163);",
                                @"&(copy|#169);",
                                @"&#(\d+);",
                                @"-->",
                                @"<!--.*\n"
                            };

            string[] Replaces ={
                                "",
                                "",
                                "\"",
                                "&",
                                "<",
                                ">",
                                " ",
                                "\xa1", //chr(161),
                                "\xa2", //chr(162),
                                "\xa3", //chr(163),
                                "\xa9", //chr(169),
                                "",
                                "\r\n",
                                ""
                               };

            string s = htmlCode;
            for (int i = 0; i < Regexs.Length; i++)
            {
                s = Regex.Replace(s, Replaces[i], Regexs[i], RegexOptions.Multiline | RegexOptions.IgnoreCase);
            }
            return s;
        }
        /// <summary>
        /// 获得纯文本
        /// </summary>
        /// <param name="htmlCode">源码</param>
        /// <returns></returns>
        public string GetTxt(string htmlCode)
        {
            string del = @"<head[^>]*>[\s\S]*?</head>";
            string content = Text.CRegex.Remove(htmlCode, del);

            del = @"(<script[^>]*>[\s\S]*?</script>)|(<IFRAME[^>]*>[\s\S]*?</IFRAME>)|(<style[^>]*>[\s\S]*?</style>|<title[^>]*>[\s\S]*?</title>|<meta[^>]*>|<option[^>]*>[\s\S]*?</option>)";
            content = Text.CRegex.Remove(content, del);
            del = @"(&nbsp;)|([\n\t]+)";
            content = Text.CRegex.Remove(content, del);
            string re = @"(<table(\s+[^>]*)*>)|(<td(\s+[^>]*)*>)|(<tr(\s+[^>]*)*>)|(<p(\s+[^>]*)*>)|(<div(\s+[^>]*)*>)|(<ul(\s+[^>]*)*>)|(<li(\s+[^>]*)*>)|</table>|</td>|</tr>|</p>|<br>|</div>|</li>|</ul>|<p />|<br />";
            content = Text.CRegex.Replace(content, re, "");
            content = Text.CRegex.Replace(content, @"[\f\n\r\v]+", "");
            content = Text.CRegex.Remove(content, @"<a(\s+[^>]*)*>[\s\S]*?</a>");
            content = Text.CRegex.Remove(content, "<[^>]+>");//去除各种HTML标记，获得纯内容
            content = content.Replace("\n", "");
            content = content.Replace("\r", "");
            content = content.Trim();
            return content;
        }

        #endregion

        #region 清除样式
        /// <summary>
        /// 清除样式 实现一键清除格式<br/>
        /// add by mashanlin 2013-07-02<br/>
        /// </summary>
        /// <param name="htmlCode">html源码</param>
        /// <param name="leaveTags">保留标签</param>
        /// <returns></returns>
        public string ClearStyle(string htmlCode, string leaveTags)
        {
            //删除脚本
            htmlCode = Regex.Replace(htmlCode, @"<script[^>]*?>.*?</script>", "", RegexOptions.IgnoreCase);
            //删除样式
            htmlCode = Regex.Replace(htmlCode, "class=['\"][^'\"]+['\"]", "", RegexOptions.IgnoreCase);
            htmlCode = Regex.Replace(htmlCode, "style=['\"][^'\"]+['\"]", "", RegexOptions.IgnoreCase);
            htmlCode = ClearHtml(htmlCode, leaveTags);

            return htmlCode;
        }
        #endregion

        #region 获取页面内容
        /// <summary>
        /// 获取网页源码
        /// </summary>
        /// <param name="url">链接</param>
        /// <returns></returns>
        public string GetHtml(string url)
        {
            return GetHtml(ref url, "");
        }
        /// <summary>
        /// 获取网页源码
        /// </summary>
        /// <param name="url">链接</param>
        /// <param name="pageEncode">文本编码:utf-8 gb2312</param>
        /// <returns></returns>
        public string GetHtml(ref string url, string pageEncode)
        {
            string content = "";
            try
            {
                HttpWebResponse response = GetResponse(url);
                if (response == null)
                {
                    return content;
                }
                url = response.ResponseUri.AbsoluteUri;
                Stream stream = response.GetResponseStream();
                byte[] buffer = GetContent(stream);
                stream.Close();
                stream.Dispose();
                string charset = "";
                if (pageEncode == null || pageEncode == "")
                {
                    //首先，从返回头信息中寻找
                    string ht = response.GetResponseHeader("Content-Type");
                    response.Close();
                    string regCharSet = "[\\s\\S]*charset=(?<charset>[\\S]*)";
                    Regex r = new Regex(regCharSet, RegexOptions.IgnoreCase);
                    Match m = r.Match(ht);
                    charset = (m.Captures.Count != 0) ? m.Result("${charset}") : "";
                    if (charset == "-8") charset = "utf-8";

                    if (charset == "")//找不到，则在文件信息本身中查找
                    {
                        //先按gb2312来获取文件信息
                        content = System.Text.Encoding.GetEncoding("gb2312").GetString(buffer);
                        regCharSet = "(<meta[^>]*charset=(?<charset>[^>'\"]*)[\\s\\S]*?>)|(xml[^>]+encoding=(\"|')*(?<charset>[^>'\"]*)[\\s\\S]*?>)";
                        r = new Regex(regCharSet, RegexOptions.IgnoreCase);
                        m = r.Match(content);
                        if (m.Captures.Count == 0)//没办法，都找不到编码，只能返回按"gb2312"获取的信息
                        {
                            return content;
                        }
                        charset = m.Result("${charset}");
                    }
                }
                else
                {
                    response.Close();
                    charset = pageEncode.ToLower();
                }
                try
                {
                    content = System.Text.Encoding.GetEncoding(charset).GetString(buffer);
                }
                catch (ArgumentException)
                {//指定的编码不可识别
                    content = System.Text.Encoding.GetEncoding("gb2312").GetString(buffer);
                }
            }
            catch
            {
                content = "";
            }
            return content;
        }

        #region 获取HTML POST/GET方式

        private int delay = 500;   //重试间隔 毫秒
        private int maxTry = 3;    //重试次数
        private int currentTry = 0;//重试次数

        /// <summary>
        /// 获取HTML GET方式
        /// </summary>
        /// <param name="url">地址</param>
        /// <param name="pageEncode">编码</param>
        /// <param name="cookieContainer">CookieContainer</param>
        public string GetHtml(string url, string pageEncode, ref CookieContainer cookieContainer)
        {
            string message = "";
            return GetHtml(url, "", false, pageEncode, ref cookieContainer, out message);
        }
        /// <summary>
        /// 获取HTML GET方式
        /// </summary>
        /// <param name="url">地址</param>
        /// <param name="pageEncode">编码</param>
        /// <param name="cookieContainer">CookieContainer</param>
        public string GetHtml(string url, string pageEncode, ref CookieContainer cookieContainer, out string message)
        {
            return GetHtml(url, "", false, pageEncode, ref cookieContainer, out message);
        }
        /// <summary>
        /// 获取HTML POST/GET方式
        /// </summary>
        /// <param name="url">地址</param>
        /// <param name="postData">POST字符串</param>
        /// <param name="isPost">是否是post</param>
        /// <param name="pageEncode">编码:UTF-8/GB2312</param>
        /// <param name="cookieContainer">CookieContainer</param>
        public string GetHtml(string url, string postData, bool isPost, string pageEncode, ref CookieContainer cookieContainer)
        {
            string message = "";
            return GetHtml(url, postData, false, pageEncode, ref cookieContainer, out message);
        }
        /// <summary>
        /// 获取HTML POST/GET方式
        /// </summary>
        /// <param name="url">地址</param>
        /// <param name="postData">post 提交的字符串</param>
        /// <param name="isPost">是否是post</param>
        /// <param name="pageEncode">编码:UTF-8/GB2312</param>
        /// <param name="cookieContainer">CookieContainer</param>
        /// <param name="message">错误信息</param>
        public string GetHtml(string url, string postData, bool isPost, string pageEncode, ref CookieContainer cookieContainer, out string message)
        {
            if (String.IsNullOrEmpty(postData))
            {
                return GetHtml(url, pageEncode, ref cookieContainer, out message);
            }
            message = "";
            url = url.Trim();
            //errMess = "";
            if (String.IsNullOrEmpty(pageEncode))
            {
                pageEncode = "gb2312";
            }
            currentTry++;
            HttpWebRequest httpRequest = null;
            HttpWebResponse httpResponse = null;
            byte[] byteRequest = Encoding.Default.GetBytes(postData);

            try
            {
                httpRequest = (HttpWebRequest)HttpWebRequest.Create(url);
                httpRequest.CookieContainer = cookieContainer;
                httpRequest.ContentType = "application/x-www-form-urlencoded";
                httpRequest.Accept = "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-shockwave-flash, application/x-silverlight, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/x-silverlight-2-b1, */*";
                httpRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022)";
                httpRequest.Method = isPost ? "POST" : "GET";
                httpRequest.ServicePoint.ConnectionLimit = 300;
                httpRequest.Referer = url;

                if (isPost)
                {
                    httpRequest.ContentLength = byteRequest.Length;
                    using (Stream stream = httpRequest.GetRequestStream())
                    {
                        stream.Write(byteRequest, 0, byteRequest.Length);
                    }
                }
                //接收返回页面
                httpResponse = (HttpWebResponse)httpRequest.GetResponse();
                string html = "";
                using (Stream responseStream = httpResponse.GetResponseStream())
                {
                    using (StreamReader streamReader = new StreamReader(responseStream, Encoding.GetEncoding(pageEncode)))
                    {
                        html = streamReader.ReadToEnd();
                    }
                }
                currentTry = 0;
                httpRequest.Abort();
                httpResponse.Close();
                return html;
            }
            catch (Exception err)
            {
                if (currentTry <= maxTry)
                {
                    System.Threading.Thread.Sleep(delay);
                    GetHtml(url, postData, isPost, pageEncode, ref cookieContainer, out message);
                }
                if (httpRequest != null) httpRequest.Abort();
                if (httpResponse != null) httpResponse.Close();
                message = err.ToString();
                currentTry--;
                return String.Empty;
            }
        }
        #endregion

        /// <summary>
        /// 获取HTTP头集合
        /// </summary>
        /// <param name="url">链接</param>
        /// <returns></returns>
        public string GetHttpHead(string url)
        {
            string sHead = "";
            Uri uri = new Uri(url);
            try
            {
                WebRequest req = WebRequest.Create(uri);
                WebResponse resp = req.GetResponse();
                WebHeaderCollection headers = resp.Headers;
                string[] sKeys = headers.AllKeys;
                foreach (string sKey in sKeys)
                {
                    sHead += sKey + ":" + headers[sKey] + "\r\n";
                }
            }
            catch
            {
            }
            return sHead;
        }

        #endregion

        #region CookieContainer转字典集合
        /// <summary>
        /// Cookie转字典集合 马山林 2014-03-12 添加
        /// </summary>
        /// <param name="cookies">CookieContainer 对象</param>
        /// <param name="url">链接</param>
        /// <returns></returns>
        public Dictionary<String, String> GetCookies(CookieContainer cookies, string url)
        {
            Dictionary<String, String> dict = null;
            CookieCollection cookie = null;
            if (cookies == null || cookies.Count == 0 || String.IsNullOrEmpty(url))
            {
                return null;
            }
            cookie = cookies.GetCookies(new Uri(url));
            try
            {
                dict = new Dictionary<string, string>();
                foreach (Cookie c in cookie)
                {
                    dict.Add(c.Name, c.Value);
                }
                cookie = null;
            }
            catch (Exception ex)
            {
                throw new Exception(ex.ToString());
            }
            return dict;
        }
        #endregion

        #region 框架页

        /// <summary>
        /// 处理框架页面问题。如果该页面是框架结构的话，返回该框架
        /// </summary>
        /// <param name="url">链接</param>
        /// <param name="htmlCode">HTML源码</param>
        /// <returns></returns>
        public string[] DealFrame(string url, string htmlCode)
        {
            string reg = @"<frame\s+[^>]*src\s*=\s*(?:""(?<src>[^""]+)""|'(?<src>[^']+)'|(?<src>[^\s>""']+))[^>]*>";
            return DealWithFrame(reg, url, htmlCode);
        }

        /// <summary>
        /// 处理浮动桢问题。如果该页面存在浮动桢，返回浮动桢
        /// </summary>
        /// <param name="url">链接</param>
        /// <param name="htmlCode">HTML源码</param>
        /// <returns></returns>
        public string[] DealIFrame(string url, string htmlCode)
        {
            string reg = @"<iframe\s+[^>]*src\s*=\s*(?:""(?<src>[^""]+)""|'(?<src>[^']+)'|(?<src>[^\s>""']+))[^>]*>";
            return DealWithFrame(reg, url, htmlCode);
        }

        #endregion

        #region 获得多个页面
        /// <summary>
        /// 获得多个页面内容
        /// </summary>
        /// <param name="listUrl">列表url</param>
        /// <param name="pageEncode">编码</param>
        /// <returns></returns>
        public List<KeyValuePair<int, string>> GetHtml(List<KeyValuePair<int, string>> listUrl, string pageEncode)
        {
            int iTimeOut = int.Parse(System.Configuration.ConfigurationManager.AppSettings["SocketTimeOut"]);
            StringBuilder sbHtml = new StringBuilder();
            List<KeyValuePair<int, string>> listResult = new List<KeyValuePair<int, string>>();
            int nBytes = 0;
            Socket sock = null;
            IPHostEntry ipHostInfo = null;
            try
            {
                // 初始化				
                Uri site = new Uri(listUrl[0].Value.ToString());
                try
                {
                    ipHostInfo = System.Net.Dns.GetHostEntry(site.Host);
                }
                catch (Exception Ex)
                {
                    throw Ex;
                }
                IPAddress ipAddress = ipHostInfo.AddressList[0];
                IPEndPoint remoteEP = new IPEndPoint(ipAddress, site.Port);
                sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
                sock.SendTimeout = iTimeOut;
                sock.ReceiveTimeout = iTimeOut;
                try
                {
                    sock.Connect(remoteEP);
                }
                catch (Exception Ex)
                {
                    throw Ex;
                }
                foreach (KeyValuePair<int, string> kvUrl in listUrl)
                {
                    site = new Uri(kvUrl.Value);
                    string sendMsg = "GET " + HttpUtility.UrlDecode(site.PathAndQuery) + " HTTP/1.1\r\n" +
                        "Accept: image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/vnd.ms-excel, application/msword, application/vnd.ms-powerpoint, */*\r\n" +
                        "Accept-Language:en-us\r\n" +
                        "Accept-Encoding:gb2312, deflate\r\n" +
                        "User-Agent: Mozilla/4.0\r\n" +
                        "Host: " + site.Host + "\r\n\r\n" + '\0';
                    // 发送
                    byte[] msg = Encoding.GetEncoding(pageEncode).GetBytes(sendMsg);
                    if ((nBytes = sock.Send(msg)) == 0)
                    {
                        sock.Shutdown(SocketShutdown.Both);
                        sock.Close();
                        return listResult;
                    }
                    // 接受
                    byte[] bytes = new byte[2048];
                    byte bt = Convert.ToByte('\x7f');
                    do
                    {
                        int count = 0;
                        try
                        {
                            nBytes = sock.Receive(bytes, bytes.Length - 1, 0);
                        }
                        catch (Exception Ex)
                        {
                            string str = Ex.Message;
                            nBytes = -1;
                        }
                        if (nBytes <= 0) break;
                        if (bytes[nBytes - 1] > bt)
                        {
                            for (int i = nBytes - 1; i >= 0; i--)
                            {
                                if (bytes[i] > bt)
                                    count++;
                                else
                                    break;
                            }
                            if (count % 2 == 1)
                            {
                                count = sock.Receive(bytes, nBytes, 1, 0);
                                if (count < 0)
                                    break;
                                nBytes = nBytes + count;
                            }
                        }
                        else
                            bytes[nBytes] = (byte)'\0';
                        string s = Encoding.GetEncoding(pageEncode).GetString(bytes, 0, nBytes);
                        sbHtml.Append(s);
                    } while (nBytes > 0);

                    listResult.Add(new KeyValuePair<int, string>(kvUrl.Key, sbHtml.ToString()));
                    sbHtml = null;
                    sbHtml = new StringBuilder();
                }
            }
            catch (Exception Ex)
            {
                string s = Ex.Message;
                try
                {
                    sock.Shutdown(SocketShutdown.Both);
                    sock.Close();
                }
                catch { }
            }
            finally
            {
                try
                {
                    sock.Shutdown(SocketShutdown.Both);
                    sock.Close();
                }
                catch { }
            }
            return listResult;
        }
        #endregion

        #region 补全链接
        /// <summary>
        /// 基于baseUrl，补全html代码中的链接 马山林 2014-3-13 添加
        /// </summary>
        /// <param name="baseUrl"></param>
        /// <param name="html"></param>
        public string FixUrl(string baseUrl, string html)
        {
            html = Regex.Replace(html, "(?is)(href|src)=(\"|\')([^(\"|\')]+)(\"|\')", (match) =>
            {
                string org = match.Value;
                string link = match.Groups[3].Value;
                if (link.StartsWith("http"))
                {
                    return org;
                }
                try
                {
                    Uri uri = new Uri(baseUrl);
                    Uri thisUri = new Uri(uri, link);
                    string fullUrl = String.Format("{0}=\"{1}\"", match.Groups[1].Value, thisUri.AbsoluteUri);
                    return fullUrl;
                }
                catch (Exception)
                {
                    return org;
                }
            });
            return html;
        }
        #endregion

        #region 获得特定内容

        /// <summary>
        /// 图片标签
        /// </summary>
        /// <param name="input">输入内容</param>
        public List<String> GetImgTags(string input)
        {
            return CRegex.GetList(input, REG_IMG, "");
        }

        /// <summary>
        /// 图片地址
        /// </summary>
        /// <param name="input">输入内容</param>
        public List<String> GetImgLinks(string input)
        {
            return CRegex.GetList(input, REG_IMG, "src");
        }

        #endregion 获得特定内容

        #region 获得文章特定项
        /// <summary>
        /// 网页标题
        /// </summary>
        public string GetTitle(string input)
        {
            return CRegex.GetText(input, REG_TITLE, "Title");
        }
        /// <summary>
        /// 获取网页body
        /// </summary>
        /// <param name="htmlCode">源码</param>
        /// <returns></returns>
        public string GetBody(string htmlCode)
        {
            if (htmlCode.Contains("<body"))
            {
                htmlCode = htmlCode.Substring(htmlCode.IndexOf("<body" + 4));
            }
            else if (htmlCode.Contains("<BODY"))
            {
                htmlCode = htmlCode.Substring(htmlCode.IndexOf("<BODY" + 4));
            }
            htmlCode = Regex.Replace(htmlCode, @"</\bbody\b[^>]*>\s*</html>", "", RegexOptions.IgnoreCase | RegexOptions.RightToLeft);
            return htmlCode;
        }
        /// <summary>
        /// 是否是XML文件
        /// </summary>
        /// <param name="htmlCode"></param>
        /// <returns></returns>
        public bool IsXml(string htmlCode)
        {
            Regex re = new Regex(@"<\x3fxml\s+", RegexOptions.IgnoreCase);
            MatchCollection mcs = re.Matches(htmlCode);
            return (mcs.Count > 0);
        }
        #endregion

        #region 查找文件
        /// <summary>
        /// 查找网页源码中文件列表 2013-12-06 mashanlin 添加
        /// </summary>
        /// <param name="url">链接 可为空</param>
        /// <param name="htmlCode">源码</param>
        /// <param name="types">文件类型，格式如：.doc|.docx|.ppt</param>
        /// <returns>附件名、附件绝对链接、类型、附件相对链接</returns>
        public List<String[]> GetFiles(string url, ref string htmlCode, string types)
        {
            return GetFiles(url, ref htmlCode, types, true);
        }
        /// <summary>
        /// 查找网页源码中文件列表 2013-05-10 mashanlin 添加
        /// </summary>
        /// <param name="url">链接</param>
        /// <param name="htmlCode">源码</param>
        /// <param name="types">文件类型，格式如：.doc|.docx|.ppt</param>
        /// <param name="isRelpaceurl">是否替换原链接</param>
        /// <returns>附件名、附件绝对链接、类型、附件相对链接</returns>
        public List<String[]> GetFiles(string url, ref string htmlCode, string types, bool isRelpaceurl)
        {
            if ((htmlCode + "").Length < 4) return null;
            List<String[]> list = new List<string[]>();
            if (String.IsNullOrEmpty(types)) types = ".docx|.doc|.pptx|.ppt|.xlsx|.xls|.txt|.pdf|.ceb|.zip|.rar|.tif|.mht|.wps";
            if (!types.Contains(".")) types = "." + types.Replace("|", ".|");

            string reg = "<a(.*?)href=['\"](?<link>[^'\"]*(?<type>" + types + ")[^'\"]*)[^>]*>(?<name>.*?)</a>";
            foreach (Match m in Regex.Matches(htmlCode, reg, RegexOptions.IgnoreCase))
            {
                string[] arr = new string[5];
                arr[0] = m.Groups["name"].Value.Trim();//文件名
                arr[1] = m.Groups["link"].Value.Trim();
                arr[1] = String.IsNullOrEmpty(url) ? arr[1] : Utility.Net.UrlHelper.AbsulateURL(arr[1], url);//文件链接
                arr[2] = m.Groups["type"].Value.Trim(); //文件类型//fileArray[1].Substring(fileArray[1].LastIndexOf('.'));
                arr[3] = m.Groups["link"].Value.Trim();
                if (isRelpaceurl)
                {
                    htmlCode = htmlCode.Replace(arr[3], arr[1]);
                }
                if (!list.Contains(arr))
                {
                    list.Add(arr);
                }
            }
            return list;
        }
        #endregion

        #region 私有方法

        private HttpWebResponse GetResponse(string url)
        {
            int timeOut = 10000;
            bool isCookie = false;
            bool isRepeat = false;
            Uri target = new Uri(url);
        ReCatch:
            try
            {
                HttpWebRequest resquest = (HttpWebRequest)WebRequest.Create(target);
                resquest.MaximumResponseHeadersLength = -1;
                resquest.ReadWriteTimeout = 120000;//120秒就超时
                resquest.Timeout = timeOut;
                resquest.MaximumAutomaticRedirections = 50;
                resquest.MaximumResponseHeadersLength = 5;
                resquest.AllowAutoRedirect = true;
                if (isCookie)
                {
                    resquest.CookieContainer = new CookieContainer();
                }
                resquest.UserAgent = "Mozilla/6.0 (compatible; MSIE 6.0; Windows NT 5.1)";
                return (HttpWebResponse)resquest.GetResponse();
            }
            catch (WebException)
            {
                if (!isRepeat)
                {
                    isRepeat = true;
                    isCookie = true;
                    goto ReCatch;
                }
                return null;
            }
            catch
            {
                return null;
            }
        }
        private byte[] GetContent(Stream stream)
        {
            ArrayList al = new ArrayList();
            try
            {
                byte[] buffer = new byte[4096];
                int count = stream.Read(buffer, 0, 4096);
                while (count > 0)
                {
                    for (int i = 0; i < count; i++)
                    {
                        al.Add(buffer[i]);
                    }
                    count = stream.Read(buffer, 0, 4096);
                }
            }
            catch { }

            return (byte[])al.ToArray(System.Type.GetType("System.Byte"));
        }
        private string[] DealWithFrame(string pattern, string url, string content)
        {
            ArrayList al = new ArrayList();
            Regex r = new Regex(pattern, RegexOptions.IgnoreCase);
            Match m = r.Match(content);
            while (m.Success)
            {
                al.Add(Net.UrlHelper.AbsulateURL(m.Groups["src"].Value, url));
                m = m.NextMatch();
            }
            return (string[])al.ToArray(System.Type.GetType("System.String"));
        }

        #endregion
    }
}
